package com.archgeek.spark.examples.v20201230

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

/**
 * 此实验是 Spark 的 WC 入门实例,可以研究清楚
 *
 * Create by pizhihui on 2020-12-30
 */
object WordCount {

  def main(args: Array[String]): Unit = {

    val conf = new SparkConf()
    conf.setAppName("word count test")
    conf.setMaster("local")
    val sc = new SparkContext(conf)


    val fileRDD: RDD[String] = sc.textFile("data/a.txt")

    val words = fileRDD.flatMap(_.split(" "))

    val wordsAndOne = words.map((_, 1))

    val res = wordsAndOne.reduceByKey(_ + _)

    val numAndOne = res.map(x => (x._2, 1))

    val resOne = numAndOne.reduceByKey(_ + _)

    resOne.foreach(println)
    res.foreach(println)

    System.in.read()

  }

}
